{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np\n",
    "from scipy.optimize import minimize\n",
    "from functools import partial\n",
    "from statsmodels.formula.api import ols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>wave</th>\n",
       "      <th>year</th>\n",
       "      <th>age</th>\n",
       "      <th>co</th>\n",
       "      <th>Y</th>\n",
       "      <th>logY</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>country</th>\n",
       "      <th>hid</th>\n",
       "      <th>pid</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">2</th>\n",
       "      <th>7601</th>\n",
       "      <th>76101</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1994.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>DK</td>\n",
       "      <td>0.829935</td>\n",
       "      <td>-0.186408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8401</th>\n",
       "      <th>84101</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1994.0</td>\n",
       "      <td>70.0</td>\n",
       "      <td>DK</td>\n",
       "      <td>1.082765</td>\n",
       "      <td>0.079518</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10701</th>\n",
       "      <th>107101</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1994.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>DK</td>\n",
       "      <td>0.770043</td>\n",
       "      <td>-0.261310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16501</th>\n",
       "      <th>165101</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1994.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>DK</td>\n",
       "      <td>1.336161</td>\n",
       "      <td>0.289800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19901</th>\n",
       "      <th>199101</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1994.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>DK</td>\n",
       "      <td>0.821379</td>\n",
       "      <td>-0.196771</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      wave    year   age  co         Y      logY\n",
       "country hid   pid                                               \n",
       "2       7601  76101    1.0  1994.0  42.0  DK  0.829935 -0.186408\n",
       "        8401  84101    1.0  1994.0  70.0  DK  1.082765  0.079518\n",
       "        10701 107101   1.0  1994.0  22.0  DK  0.770043 -0.261310\n",
       "        16501 165101   1.0  1994.0  46.0  DK  1.336161  0.289800\n",
       "        19901 199101   1.0  1994.0  23.0  DK  0.821379 -0.196771"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_pickle('../data_sources/echp/echp_incomes.pkl')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wave</th>\n",
       "      <th>year</th>\n",
       "      <th>age</th>\n",
       "      <th>Y</th>\n",
       "      <th>logY</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>48395.000000</td>\n",
       "      <td>48395.000000</td>\n",
       "      <td>48395.000000</td>\n",
       "      <td>48395.000000</td>\n",
       "      <td>48395.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>4.340469</td>\n",
       "      <td>1999.639054</td>\n",
       "      <td>57.623473</td>\n",
       "      <td>1.014240</td>\n",
       "      <td>-0.118761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>2.304796</td>\n",
       "      <td>7.926220</td>\n",
       "      <td>18.380789</td>\n",
       "      <td>0.528584</td>\n",
       "      <td>0.538880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1994.000000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>0.057038</td>\n",
       "      <td>-2.864029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>1995.000000</td>\n",
       "      <td>40.000000</td>\n",
       "      <td>0.644961</td>\n",
       "      <td>-0.438565</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>4.000000</td>\n",
       "      <td>1997.000000</td>\n",
       "      <td>62.000000</td>\n",
       "      <td>0.906191</td>\n",
       "      <td>-0.098505</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>6.000000</td>\n",
       "      <td>1999.000000</td>\n",
       "      <td>73.000000</td>\n",
       "      <td>1.258058</td>\n",
       "      <td>0.229569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>8.000000</td>\n",
       "      <td>2021.000000</td>\n",
       "      <td>85.000000</td>\n",
       "      <td>3.519351</td>\n",
       "      <td>1.258277</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               wave          year           age             Y          logY\n",
       "count  48395.000000  48395.000000  48395.000000  48395.000000  48395.000000\n",
       "mean       4.340469   1999.639054     57.623473      1.014240     -0.118761\n",
       "std        2.304796      7.926220     18.380789      0.528584      0.538880\n",
       "min        1.000000   1994.000000     25.000000      0.057038     -2.864029\n",
       "25%        2.000000   1995.000000     40.000000      0.644961     -0.438565\n",
       "50%        4.000000   1997.000000     62.000000      0.906191     -0.098505\n",
       "75%        6.000000   1999.000000     73.000000      1.258058      0.229569\n",
       "max        8.000000   2021.000000     85.000000      3.519351      1.258277"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df[(df['age']>=25) & (df['age']<=85)]\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vech(cov):\n",
    "\tT = cov.shape[0]\n",
    "\tv = []\n",
    "\tfor i in range(T):\n",
    "\t\tfor j in range(i,T):\n",
    "\t\t\tv.append(cov[j,i])\n",
    "\treturn np.array(v)\n",
    "def mdfunc(theta,cov,weight):\n",
    "\trho = theta[0]\n",
    "\tsige = theta[1]\n",
    "\tsigv = theta[2]\n",
    "\tT = cov.shape[0]\n",
    "\tcov_true = np.zeros((T,T))\n",
    "\tfor i in range(T):\n",
    "\t\tfor j in range(T):\n",
    "\t\t\tcov_true[i,j] = (i==j)*sigv + (rho**(np.abs(i-j)))*sige/(1.0-rho**2)\n",
    "\td = vech(cov) - vech(cov_true)\n",
    "\tfunc = d.reshape((1,len(d))) @ weight @ d.reshape((len(d),1))\n",
    "\treturn func[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(seed=1234)\n",
    "def get_cov(data,reps):\n",
    "\tT = len(data.columns)\n",
    "\tcov = data.cov().to_numpy()\n",
    "\tvcov = vech(cov)\n",
    "\tweight = vcov.reshape((vcov.shape[0],1))@vcov.reshape((1,vcov.shape[0]))\n",
    "\tweight = np.zeros((weight.shape[0],weight.shape[1]))\n",
    "\tfor r in range(reps):\n",
    "\t\trcov = data.sample(n=len(data),replace=True).cov().to_numpy()\n",
    "\t\tvrcov = vech(rcov)\n",
    "\t\tweight += vrcov.reshape((vrcov.shape[0],1))@vrcov.reshape((1,vrcov.shape[0]))\n",
    "\tweight = np.linalg.inv(weight/reps)\n",
    "\treturn cov, weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_pars(co):\n",
    "\tdata = df.loc[(df['co']==co) & (df['logY'].isna()==False),:]\n",
    "\tdata = data.reset_index()\n",
    "\tdata = data.loc[:,['pid','wave','age','logY']]\n",
    "\tmodel = ols('logY ~ C(age)',data=data).fit()\n",
    "\tdata.loc[:,'e'] = model.resid.to_list()\n",
    "\tdata = data.loc[:,['pid','wave','e']]\n",
    "\tdata = pd.pivot(data,index='pid',columns='wave')\n",
    "\tcov  = data.cov().to_numpy()\n",
    "\tweight = np.eye(vech(cov).shape[0])\n",
    "\tf = partial(mdfunc,cov=cov,weight=weight)\n",
    "\titheta = np.array([0.9,0.02,0.02])\n",
    "\topt = minimize(f,x0=itheta)\n",
    "\treturn opt.x\n",
    "\t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DE</th>\n",
       "      <th>DK</th>\n",
       "      <th>FR</th>\n",
       "      <th>NL</th>\n",
       "      <th>IT</th>\n",
       "      <th>SP</th>\n",
       "      <th>SE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>rho</th>\n",
       "      <td>0.928252</td>\n",
       "      <td>0.934175</td>\n",
       "      <td>0.966967</td>\n",
       "      <td>0.939743</td>\n",
       "      <td>0.912178</td>\n",
       "      <td>0.972280</td>\n",
       "      <td>0.934175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sige</th>\n",
       "      <td>0.029244</td>\n",
       "      <td>0.011039</td>\n",
       "      <td>0.012231</td>\n",
       "      <td>0.017194</td>\n",
       "      <td>0.032484</td>\n",
       "      <td>0.010824</td>\n",
       "      <td>0.011039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sigv</th>\n",
       "      <td>0.055352</td>\n",
       "      <td>0.073201</td>\n",
       "      <td>0.081796</td>\n",
       "      <td>0.086212</td>\n",
       "      <td>0.096826</td>\n",
       "      <td>0.085114</td>\n",
       "      <td>0.073201</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            DE        DK        FR        NL        IT        SP        SE\n",
       "rho   0.928252  0.934175  0.966967  0.939743  0.912178  0.972280  0.934175\n",
       "sige  0.029244  0.011039  0.012231  0.017194  0.032484  0.010824  0.011039\n",
       "sigv  0.055352  0.073201  0.081796  0.086212  0.096826  0.085114  0.073201"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "countries = ['DE','DK','FR','NL','IT','SP']\n",
    "table = pd.DataFrame(index=['rho','sige','sigv'],columns=countries)\n",
    "for co in countries:\n",
    "\ttable[co] = get_pars(co)\n",
    "table['SE'] = table.loc[:,'DK']\n",
    "table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DE</th>\n",
       "      <th>DK</th>\n",
       "      <th>FR</th>\n",
       "      <th>IT</th>\n",
       "      <th>NL</th>\n",
       "      <th>SE</th>\n",
       "      <th>SP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>rho</th>\n",
       "      <td>0.928252</td>\n",
       "      <td>0.934175</td>\n",
       "      <td>0.966967</td>\n",
       "      <td>0.912178</td>\n",
       "      <td>0.939743</td>\n",
       "      <td>0.934175</td>\n",
       "      <td>0.972280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sige</th>\n",
       "      <td>0.029244</td>\n",
       "      <td>0.011039</td>\n",
       "      <td>0.012231</td>\n",
       "      <td>0.032484</td>\n",
       "      <td>0.017194</td>\n",
       "      <td>0.011039</td>\n",
       "      <td>0.010824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sigv</th>\n",
       "      <td>0.055352</td>\n",
       "      <td>0.073201</td>\n",
       "      <td>0.081796</td>\n",
       "      <td>0.096826</td>\n",
       "      <td>0.086212</td>\n",
       "      <td>0.073201</td>\n",
       "      <td>0.085114</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            DE        DK        FR        IT        NL        SE        SP\n",
       "rho   0.928252  0.934175  0.966967  0.912178  0.939743  0.934175  0.972280\n",
       "sige  0.029244  0.011039  0.012231  0.032484  0.017194  0.011039  0.010824\n",
       "sigv  0.055352  0.073201  0.081796  0.096826  0.086212  0.073201  0.085114"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table = table.loc[:,['DE','DK','FR','IT','NL','SE','SP']]\n",
    "table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>co</th>\n",
       "      <th>pid</th>\n",
       "      <th>year</th>\n",
       "      <th>age</th>\n",
       "      <th>Y</th>\n",
       "      <th>logY</th>\n",
       "      <th>wave</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>US</td>\n",
       "      <td>4001</td>\n",
       "      <td>1989</td>\n",
       "      <td>67.0</td>\n",
       "      <td>0.720443</td>\n",
       "      <td>-0.327888</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>US</td>\n",
       "      <td>4003</td>\n",
       "      <td>1989</td>\n",
       "      <td>38.0</td>\n",
       "      <td>0.900614</td>\n",
       "      <td>-0.104679</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>US</td>\n",
       "      <td>4008</td>\n",
       "      <td>1989</td>\n",
       "      <td>24.0</td>\n",
       "      <td>0.684175</td>\n",
       "      <td>-0.379542</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>US</td>\n",
       "      <td>4170</td>\n",
       "      <td>1989</td>\n",
       "      <td>34.0</td>\n",
       "      <td>0.789620</td>\n",
       "      <td>-0.236204</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>US</td>\n",
       "      <td>4172</td>\n",
       "      <td>1989</td>\n",
       "      <td>40.0</td>\n",
       "      <td>1.564385</td>\n",
       "      <td>0.447493</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    co   pid  year   age         Y      logY  wave\n",
       "0   US  4001  1989  67.0  0.720443 -0.327888     1\n",
       "2   US  4003  1989  38.0  0.900614 -0.104679     1\n",
       "6   US  4008  1989  24.0  0.684175 -0.379542     1\n",
       "15  US  4170  1989  34.0  0.789620 -0.236204     1\n",
       "16  US  4172  1989  40.0  1.564385  0.447493     1"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_pickle('../data_sources/psid/psid_incomes.pkl')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pid</th>\n",
       "      <th>year</th>\n",
       "      <th>age</th>\n",
       "      <th>Y</th>\n",
       "      <th>logY</th>\n",
       "      <th>wave</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>6.326700e+04</td>\n",
       "      <td>63267.000000</td>\n",
       "      <td>63267.000000</td>\n",
       "      <td>63267.000000</td>\n",
       "      <td>63267.000000</td>\n",
       "      <td>63267.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.499166e+06</td>\n",
       "      <td>1993.040448</td>\n",
       "      <td>45.861792</td>\n",
       "      <td>0.985992</td>\n",
       "      <td>-0.279632</td>\n",
       "      <td>5.040448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>2.304756e+06</td>\n",
       "      <td>2.519332</td>\n",
       "      <td>15.300520</td>\n",
       "      <td>0.679334</td>\n",
       "      <td>0.794749</td>\n",
       "      <td>2.519332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>4.001000e+03</td>\n",
       "      <td>1989.000000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>0.032046</td>\n",
       "      <td>-3.440590</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.398003e+06</td>\n",
       "      <td>1991.000000</td>\n",
       "      <td>34.000000</td>\n",
       "      <td>0.473577</td>\n",
       "      <td>-0.747441</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.665006e+06</td>\n",
       "      <td>1993.000000</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>0.844432</td>\n",
       "      <td>-0.169092</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>5.814034e+06</td>\n",
       "      <td>1995.000000</td>\n",
       "      <td>56.000000</td>\n",
       "      <td>1.344353</td>\n",
       "      <td>0.295913</td>\n",
       "      <td>7.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>6.872177e+06</td>\n",
       "      <td>1997.000000</td>\n",
       "      <td>85.000000</td>\n",
       "      <td>4.048423</td>\n",
       "      <td>1.398328</td>\n",
       "      <td>9.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                pid          year           age             Y          logY  \\\n",
       "count  6.326700e+04  63267.000000  63267.000000  63267.000000  63267.000000   \n",
       "mean   3.499166e+06   1993.040448     45.861792      0.985992     -0.279632   \n",
       "std    2.304756e+06      2.519332     15.300520      0.679334      0.794749   \n",
       "min    4.001000e+03   1989.000000     25.000000      0.032046     -3.440590   \n",
       "25%    1.398003e+06   1991.000000     34.000000      0.473577     -0.747441   \n",
       "50%    2.665006e+06   1993.000000     42.000000      0.844432     -0.169092   \n",
       "75%    5.814034e+06   1995.000000     56.000000      1.344353      0.295913   \n",
       "max    6.872177e+06   1997.000000     85.000000      4.048423      1.398328   \n",
       "\n",
       "               wave  \n",
       "count  63267.000000  \n",
       "mean       5.040448  \n",
       "std        2.519332  \n",
       "min        1.000000  \n",
       "25%        3.000000  \n",
       "50%        5.000000  \n",
       "75%        7.000000  \n",
       "max        9.000000  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.loc[(df['age']>=25) & (df['age']<=85),:]\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "pars = get_pars('US')\n",
    "table.loc[:,'US'] = pars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DE</th>\n",
       "      <th>DK</th>\n",
       "      <th>FR</th>\n",
       "      <th>IT</th>\n",
       "      <th>NL</th>\n",
       "      <th>SE</th>\n",
       "      <th>SP</th>\n",
       "      <th>US</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>rho</th>\n",
       "      <td>0.928252</td>\n",
       "      <td>0.934175</td>\n",
       "      <td>0.966967</td>\n",
       "      <td>0.912178</td>\n",
       "      <td>0.939743</td>\n",
       "      <td>0.934175</td>\n",
       "      <td>0.972280</td>\n",
       "      <td>0.958145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sige</th>\n",
       "      <td>0.029244</td>\n",
       "      <td>0.011039</td>\n",
       "      <td>0.012231</td>\n",
       "      <td>0.032484</td>\n",
       "      <td>0.017194</td>\n",
       "      <td>0.011039</td>\n",
       "      <td>0.010824</td>\n",
       "      <td>0.037235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sigv</th>\n",
       "      <td>0.055352</td>\n",
       "      <td>0.073201</td>\n",
       "      <td>0.081796</td>\n",
       "      <td>0.096826</td>\n",
       "      <td>0.086212</td>\n",
       "      <td>0.073201</td>\n",
       "      <td>0.085114</td>\n",
       "      <td>0.118764</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sigs</th>\n",
       "      <td>0.211379</td>\n",
       "      <td>0.086705</td>\n",
       "      <td>0.188239</td>\n",
       "      <td>0.193439</td>\n",
       "      <td>0.147100</td>\n",
       "      <td>0.086705</td>\n",
       "      <td>0.197983</td>\n",
       "      <td>0.454320</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            DE        DK        FR        IT        NL        SE        SP  \\\n",
       "rho   0.928252  0.934175  0.966967  0.912178  0.939743  0.934175  0.972280   \n",
       "sige  0.029244  0.011039  0.012231  0.032484  0.017194  0.011039  0.010824   \n",
       "sigv  0.055352  0.073201  0.081796  0.096826  0.086212  0.073201  0.085114   \n",
       "sigs  0.211379  0.086705  0.188239  0.193439  0.147100  0.086705  0.197983   \n",
       "\n",
       "            US  \n",
       "rho   0.958145  \n",
       "sige  0.037235  \n",
       "sigv  0.118764  \n",
       "sigs  0.454320  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table.loc['sigs',:] = table.loc['sige',:]/(1.0-table.loc['rho',:]**2)\n",
    "table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "table.round(3).to_latex('../tables/table_5_income_shocks.tex')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = table.loc[['rho','sige'],:]\n",
    "params.to_pickle('../model/params/income_shocks.pkl')"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "cf2a50979671a58939829e6829efb726aa5da11149213b77bd50351f899d04fb"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
